import numpy as np
import pandas as pd
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from scipy.optimize import minimize
from sklearn import metrics
df = pd.read_csv('ex2/ex2data1.txt', sep=",", header=None)
df.columns = ['exam1','exam2','admission']
fig = px.scatter(df,x='exam1',y='exam2',color='admission',symbol='admission',
color_continuous_scale=px.colors.sequential.Jet)
fig.update_layout(legend=dict(
yanchor="top",
y=0.99,
xanchor="left",
x=0.01
))
fig.update(layout_coloraxis_showscale=False)
fig.show()
We can clearly see that this dataset is devisable into two groups
First we will implement logistic regression by hand
And afterwards with prebuilt libraries
def sigmoid(z):
return 1 / (1 + np.exp(-z) )
y = df.iloc[:,-1].to_numpy()
x = df.iloc[:,:-1]
m, n = x.shape
x = np.concatenate([np.ones((m, 1)), x], axis=1)
def costFunction(theta, X, y):
m = y.size
J = 0
grad = np.zeros(theta.shape)
h = sigmoid(X.dot(theta.T))
J =-(1/m) * (y.dot(np.log(h))+(1-y).dot(np.log(1-h)))
grad = (1/m) * (h-y).dot(X)
return J, grad
initial_theta = np.array([0,0,0])
res = minimize(costFunction,
initial_theta,
(x, y),
jac=True,
method='TNC',
options={'maxiter': 10000})
cost = res.fun
theta = res.x
x_ = np.linspace(x[:,1:].min(), x[:,1:].max(),2)
y_ = -(theta[0] + x_*theta[1])/theta[2]
fig.add_trace(
go.Scatter(
x=x_,
y=y_,
mode='lines',
line={"color":"black"},
name="matrix_solution"
))
fig.show()
df['predicted']=sigmoid(x.dot(theta)).round(0)
metrics.confusion_matrix(df['admission'],df['predicted'])
array([[34, 6],
[ 5, 55]], dtype=int64)
confusion = pd.pivot_table(df.groupby(['admission','predicted']).count()['exam1'].to_frame(),index='admission',columns='predicted',)
from sklearn.linear_model import LogisticRegression
m = LogisticRegression()
m.fit(df.iloc[:,:2],df.iloc[:,-1])
LogisticRegression()
y_sci = -(m.intercept_ + x_*m.coef_[0][0])/m.coef_[0][1]
fig.add_trace(
go.Scatter(
x=x_,
y=y_sci,
mode='lines',
line={"color":"green"},
name="plotly_solution"
))
fig.update_layout(legend=dict(
orientation="h",
yanchor="bottom",
y=1.02,
xanchor="right",
x=1
))
fig.show()
df2 = pd.read_csv('ex2/ex2data2.txt', sep=",", header=None)
df2.columns = ['test1','test2','result']
df2.sample(5)
| test1 | test2 | result | |
|---|---|---|---|
| 50 | 0.13767 | 0.575290 | 1 |
| 89 | -0.38076 | 0.918860 | 0 |
| 36 | -0.23675 | -0.214180 | 1 |
| 99 | 0.82316 | 0.663010 | 0 |
| 108 | -0.61118 | -0.067982 | 0 |
fig2 = px.scatter(df2,x='test1',y='test2',color='result',symbol='result',
color_continuous_scale=px.colors.sequential.Jet,
width=600, height=600)
fig2.update_layout(legend=dict(
yanchor="top",
y=0.99,
xanchor="left",
x=0.01
))
fig2.update(layout_coloraxis_showscale=False)
fig2.show()
from sklearn.preprocessing import PolynomialFeatures
pol_6 = PolynomialFeatures(6)
feature_df = pd.DataFrame(pol_6.fit_transform(df2.iloc[:,:-1]))
y = df2.iloc[:,-1]
def costFunction_reg(theta, X, y, lamb = 1):
m = y.size
J = 0
grad = np.zeros(theta.shape)
h = sigmoid(X.dot(theta.T))
temp_theta = theta
temp_theta[0] = 0 # we do not regularize bias
J =-(1/m) * (y.dot(np.log(h))+(1-y).dot(np.log(1-h))) + (lamb/(2*m))*np.sum(np.square(temp_theta))
grad = (1/m) * (h-y).dot(X) + (lamb/m) * temp_theta
return J, grad
initial_theta = np.zeros(feature_df.shape[1])
X = feature_df
cost, grad = costFunction_reg(initial_theta, X, y, 1)
cost
0.6931471805599453
def generate_solution_with_lambda(lambda_ = 0):
initial_theta = np.zeros(X.shape[1])
options= {'maxiter': 500}
res = minimize(costFunction_reg,
initial_theta,
(X, y, lambda_),
jac=True,
method='TNC',
options=options)
cost = res.fun
theta = res.x
return cost, theta
def plot_solution(df,theta):
df_ = df
x_min, x_max = min(df_['test1']), max(df_['test1'])
y_min, y_max = min(df_['test2']), max(df_['test2'])
j = np.linspace(x_min, x_max, 100)
k = np.linspace(y_min, y_max, 100)
z = np.zeros((j.size, k.size))
# Evaluate z = theta*x over the grid
for i, ji in enumerate(j):
for ii, kii in enumerate(k):
z[i, ii] = np.dot(pol_6.fit_transform(np.array([[ji,kii]])), theta)
z = z.T
return j,k,z
fig3 = go.Figure(fig2)
lambdas = [1]
for l in lambdas:
c,t = generate_solution_with_lambda(l)
j,k,z = plot_solution(df2,t)
fig3.add_trace(
go.Contour(
z=z,
x=j,
y=k,
contours=dict(
start=0.5,
end=0.5,
size=1,
),
contours_coloring='lines',
line_width=2,
colorscale='Reds',
name = f'Lambda: {l}'
)
)
fig3.update_layout(legend=dict(
orientation="h",
yanchor="bottom",
y=1.02,
xanchor="right",
x=1
))
fig3.show()
elastic_log_reg = LogisticRegression(penalty='elasticnet',solver = 'saga',max_iter=500,l1_ratio=0.5,fit_intercept=False)
clf = elastic_log_reg.fit(feature_df,y)
clf.score(X,y)
0.8305084745762712
sci_theta = clf.coef_[0]
fig3 = go.Figure(fig2)
j,k,z = plot_solution(df2,sci_theta)
fig3.add_trace(
go.Contour(
z=z,
x=j,
y=k,
contours=dict(
start=0.5,
end=0.5,
size=1,
),
contours_coloring='lines',
line_width=2,
colorscale='Reds',
name = 'Scikit boundry'
)
)
fig3.update_layout(legend=dict(
orientation="h",
yanchor="bottom",
y=1.02,
xanchor="right",
x=1
))
# fig3.data = (fig3.data[1],fig3.data[0])
fig3.show()